﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net;
using System.IO;
using System.Text.RegularExpressions;

namespace Orchid
{
    public class WebPage
    {


        #region 构造函数

        public WebPage(string url)
        {
            URL = url;
            //Thread thread = new Thread(new ThreadStart(GetContent));
            //thread.Start();
            GetContent();
            GetMainContent();
        }

        #endregion

        #region 私有方法

        void GetContent()
        {

            #region Request

            HttpWebRequest request = HttpWebRequest.Create(URL) as HttpWebRequest;
            request.AllowAutoRedirect = true;
            request.AllowWriteStreamBuffering = true;
            request.Referer = "";
            request.Timeout = 120 * 1000;

            #endregion

            #region Response

            HttpWebResponse response = request.GetResponse() as HttpWebResponse;
            if (response.StatusCode == HttpStatusCode.OK)
            {
                string characterSet = response.CharacterSet;
                Encoding encode;
                switch (characterSet)
                {
                    case "": encode = Encoding.Default; break;
                    case "ISO-8859-1": encode = Encoding.GetEncoding("gb2312"); break;
                    default: encode = Encoding.GetEncoding(characterSet); break;
                }

                Stream receiveStream = response.GetResponseStream();

                StreamReader reader = new StreamReader(receiveStream, encode);
                while (reader.Peek() > -1)
                {
                    Content += reader.ReadLine();
                }

                //从解析出的字符串里判断charset，如果和http应答的编码不一直
                //那么以页面声明的为准，再次从内存流里重新读取文本
                //////////Regex reg =
                //////////    new Regex(@"<meta[\s\S]+?charset=(.*)""[\s\S]+?>",
                //////////              RegexOptions.Multiline | RegexOptions.IgnoreCase);
                //////////MatchCollection mc = reg.Matches(_content);
                //////////if (mc.Count > 0)
                //////////{
                //////////    string tempCharSet = mc[0].Result("$1");
                //////////    if (string.Compare(tempCharSet, characterSet, true) != 0)
                //////////    {
                //////////        encode = Encoding.GetEncoding(tempCharSet);
                //////////        _content = string.Empty;
                //////////        receiveStream.Seek(0, SeekOrigin.Begin);
                //////////        reader = new StreamReader(receiveStream, encode);
                //////////        while (reader.Peek() > -1)
                //////////        {
                //////////            _content += reader.ReadLine();
                //////////        }
                //////////    }
                //////////}
                reader.Dispose();
                receiveStream.Dispose();
            }
            else
            {
                Content = "请求失败,请检查网络";
            }
            response.Close();

            #endregion

        }

        void GetMainContent()
        {
            string reg1 = @"<(p|br)[^<]*>";
            string reg2 = @"(\[([^=]*)(=[^\]]*)?\][\s\S]*?\[/\1\])|(?<lj>(?=[^\u4E00-\u9FA5\uFE30-\uFFA0,."");])<a\s+[^>]*>[^<]{2,}</a>(?=[^\u4E00-\u9FA5\uFE30-\uFFA0,."");]))|(?<Style><style[\s\S]+?/style>)|(?<select><select[\s\S]+?/select>)|(?<Script><script[\s\S]*?/script>)|(?<Explein><\!\-\-[\s\S]*?\-\->)|(?<li><li(\s+[^>]+)?>[\s\S]*?/li>)|(?<Html></?\s*[^> ]+(\s*[^=>]+?=['""]?[^""']+?['""]?)*?[^\[<]*>)|(?<Other>&[a-zA-Z]+;)|(?<Other2>\#[a-z0-9]{6})|(?<Space>\s+)|(\&\#\d+\;)";

            string input = Content;
            //1、获取网页的所有div标签
            List<string> list = GetTags(input, "div");

            //2、去除汉字少于200字的div
            List<string> needToRemove = new List<string>();
            foreach (string s in list)
            {
                Regex r = new Regex("[\u4e00-\u9fa5]");
                if (r.Matches(s).Count < 300)
                {
                    needToRemove.Add(s);
                }
            }
            foreach (string s in needToRemove)
            {
                list.Remove(s);
            }

            //3、把剩下的div按汉字比例多少倒序排列,
            list.Sort(CompareDinosByChineseLength);
            if (list.Count < 1)
            {
                MainContent = "";
            }
            if (list.Count > 0)
            {
                input = list[list.Count - 1];
            }
            //4、把p和br替换成特殊的占位符[p][br]
            input = new Regex(reg1, RegexOptions.Multiline | RegexOptions.IgnoreCase).Replace(input, "[$1]");

            //5、去掉HTML标签，保留汉字
            input = new Regex(reg2, RegexOptions.Multiline | RegexOptions.IgnoreCase).Replace(input, "");

            //6、把特殊占维护替换成回车和换行
            input = new Regex("\\[p]", RegexOptions.Multiline | RegexOptions.IgnoreCase).Replace(input, "\r\n　　");
            input = new Regex("\\[br]", RegexOptions.Multiline | RegexOptions.IgnoreCase).Replace(input, "\r\n");
            MainContent = input;
        }

        #region 判断两段文本里哪个中文占的比例高

        /// <summary>
        /// 判断两段文本里哪个中文占的比例高
        /// </summary>
        /// <param name="x"></param>
        /// <param name="y"></param>
        /// <returns></returns>
        public static int CompareDinosByChineseLength(string x, string y)
        {
            if (x == null)
            {
                if (y == null)
                {
                    return 0;
                }
                else
                {
                    return -1;
                }
            }
            else
            {
                if (y == null)
                {
                    return 1;
                }
                else
                {
                    Regex r = new Regex("[\u4e00-\u9fa5]");
                    float xCount = (float)(r.Matches(x).Count) / (float)x.Length;
                    float yCount = (float)(r.Matches(y).Count) / (float)y.Length;

                    int retval = xCount.CompareTo(yCount);

                    if (retval != 0)
                    {
                        return retval;
                    }
                    else
                    {
                        return x.CompareTo(y);
                    }
                }
            }
        }

        #endregion



        /// <summary>
        /// 获取一个网页源码中的标签列表，支持嵌套，一般或去div，td等容器
        /// </summary>
        /// <param name="input"></param>
        /// <param name="tag"></param>
        /// <returns></returns>
        public static List<string> GetTags(string input, string tag)
        {
            StringReader strReader = new StringReader(input);
            int lowerThanCharCounter = 0;
            int lowerThanCharPos = 0;
            Stack<int> tagPos = new Stack<int>();
            List<string> taglist = new List<string>();
            int i = 0;
            while (true)
            {
                try
                {
                    int intCharacter = strReader.Read();
                    if (intCharacter == -1) break;

                    char convertedCharacter = Convert.ToChar(intCharacter);

                    if (lowerThanCharCounter > 0)
                    {
                        if (convertedCharacter == '>')
                        {
                            lowerThanCharCounter--;

                            string biaoqian = input.Substring(lowerThanCharPos, i - lowerThanCharPos + 1);
                            if (biaoqian.StartsWith(string.Format("<{0}", tag)))
                            {
                                tagPos.Push(lowerThanCharPos);
                            }
                            if (biaoqian.StartsWith(string.Format("</{0}", tag)))
                            {
                                if (tagPos.Count < 1)
                                    continue;
                                int tempTagPos = tagPos.Pop();
                                string strdiv = input.Substring(tempTagPos, i - tempTagPos + 1);
                                taglist.Add(strdiv);
                            }
                        }
                    }

                    if (convertedCharacter == '<')
                    {
                        lowerThanCharCounter++;
                        lowerThanCharPos = i;
                    }
                }
                finally
                {
                    i++;
                }
            }
            return taglist;
        }

        #endregion

        #region 属性

        public string URL
        {
            get;
            private set;
        }
        public string Content
        {
            get;
            private set;
        }
        public string MainContent
        {
            get;
            private set;
        }

        #endregion

    }
}
